import requests
from bs4 import BeautifulSoup

# 定义基本 URL 和要爬取的页面数量
base_url = 'https://gsgs3.com/MOVIE03/list_'
num_pages = 87  # 假设要爬取前三页

all_url = []

# 定义一个函数来获取页面内容并提取视频预览
def get_video_previews(url:str):
    response = requests.get(url)
    # with open(url.split('.')[-2].split('/')[-1]+'.html', 'wb') as fw:
    #     fw.write(response.content)

    if response.status_code == 200:
        soup = BeautifulSoup(response.content, 'html.parser')
        # 这里根据具体页面结构来定位视频预览的元素，然后提取相关信息
        all_dl = soup.find('div', class_='row col5 clearfix').find_all('dl')
        for dl in all_dl:
            # 这里可以提取视频的标题、链接、缩略图等信息
            all_url.append([dl.find('dt').find('a')['title'], dl.find('dt').find('a')['href']])
    else:
        print(f'Failed to retrieve page: {url}')

# 循环遍历爬取指定数量的页面
for i in range(1, num_pages + 1):
    url = f'{base_url}{i}.html'
    print(f'Fetching page: {url}')
    get_video_previews(url)

with open('1.txt', 'w', encoding='utf-8') as s:
    for line in all_url:
        s.write(line[0])
        s.write(', ')
        s.write(line[1])
        s.write('\n')
